from bs4 import BeautifulSoup
import requests

# 示例爬取百度首页
url = 'https://www.baidu.com'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
}
# 发送请求并获取响应
response = requests.get(url, headers=headers)

# 解析HTML内容
soup = BeautifulSoup(response.content, 'html.parser')
# 查找需要的信息
title = soup.find('title').text
divs = soup.findAll('span', class_='title-content-title')
lis = soup.findAll('li', class_='hotsearch-item')
result = []
for value in lis:
    a = value.find('a')['href']
    result.append({'text': value.text, 'url': a})
print(result)
